# -*- coding: utf-8 -*-

import json
import scrapy
from time import time
from scrapy.http import Request
from toutiao.items import ToutiaoItem


class ToutiaoSpider(scrapy.Spider):
    name = 'toutiao'
    # allowed_domains = ['https://www.toutiao.com/']
    # api_url = 'http://www.toutiao.com/api/pc/feed/?category=news_hot&utm_source=toutiao&widen=1&max_behot_time={}&max_behot_time_tmp={}'
    api_url = 'http://www.toutiao.com/api/pc/feed/?max_behot_time={}&category=__all__&utm_source=toutiao&tadrequire=true'
    # start_urls = [api_url.format(int(time()), int(time())), ]
    start_urls = [api_url.format(int(time())), ]

    headers = {'Host': 'www.toutiao.com',
              'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:56.0) Gecko/20100101 Firefox/56.0',
              'X-Requested-With': 'XMLHttpRequest',
              'Referer': 'http://www.toutiao.com/',
              'Connection': 'keep-alive'}

    def parse(self, response):
        toutiaoItem = ToutiaoItem()

        page_data = response.text
        page_json = json.loads(page_data)

        json_data = page_json['data']
        for data in json_data:
            if data.get('tag') != 'ad':
                group_id = data.get('group_id')
                title = data.get('title', 'null')
                label = data.get('label', 'null')
                source = data.get('source', 'null')
                chinese_tag = data.get('chinese_tag', 'null')
                link = 'http://www.toutiao.com{}'.format(data.get('source_url'))
                abstract = data.get('abstract', 'null')
                article_genre = data.get('article_genre', 'null')
                comments_count = data.get('comments_count', 'null')

                toutiaoItem['group_id'] = str(group_id)
                toutiaoItem['title'] = title
                toutiaoItem['label'] = label
                toutiaoItem['source'] = source
                toutiaoItem['chinese_tag'] = chinese_tag
                toutiaoItem['link'] = link
                toutiaoItem['abstract'] = abstract
                toutiaoItem['article_genre'] = article_genre
                toutiaoItem['comments_count'] = comments_count
                yield toutiaoItem

        next_time = (page_json['next'].get('max_behot_time') if page_json.get('next') else None)
        if next_time:
            # url = self.api_url.format(next_time, next_time)
            url = self.api_url.format(next_time)
            yield Request(url=url, headers=self.headers, callback=self.parse)

